# -*- coding: ISO-8859-1 -*-
"""\
Convert data from the IEEEXplore web site to BibTeX format

"""

# Copyright (c) 2006 Kjell Magne Fauske
# See LICENSE for details.

import re
import bibtexbib
import converterbase
from biblibutils import *
from sets import Set

from ev2parser import _monthmap
# import ieeedata

# ^\s*(?P<title>.*?)$\s*^(.*?)$\s*^(.*?)$\s*(.*?)Abstract(.*?)Index


# Todo: The dataext regexp have to be more robust!!
# Todo: App will hang if EV2 data is uses !!!!

# Regular expressions are used for extracting data from the copied text. 
# I'm no regexp expert, so there is probably much room for improvement. 

_dataext = r"""
(
    \s* ^(?P<title>.*?)$
    \s* ^(?P<authors>.*?)$   
    \s* ^(?P<address>.*?)$
    ^$
    \s* (?P<datapart>.*?)
    ^Abstract$(?P<abstract>.*?) Index[ ]Terms
    #.*?Controlled[ ]Indexing
    #(?P<ci>.*)Non[-]controlled[ ]Indexing
    #.*? (?P<nci>.*)Author[ ]Keywords
)
"""

# have to put the index term extraction in an other regexp in case
# they are not available
_indextermsext = r"""
(
    ^Index[ ]Terms
    .*?Controlled[ ]Indexing
    (?P<ci>.*)Non[-]controlled[ ]Indexing
    .*? (?P<nci>.*)Author[ ]Keywords
)
"""

_dataext_re = re.compile(_dataext,
                            re.VERBOSE |re.DOTALL | re.MULTILINE)

_indextermsext_re = re.compile(_indextermsext,
                            re.VERBOSE | re.DOTALL | re.MULTILINE)

# regex for splitting each line into tag and data field
_tagdata_re = re.compile(r"^(.*?):[ \t]*(.*?)$", 
                             re.MULTILINE)



# There's a difference between the search view and the
# "View Selected Items" view. 
# Search view:
#   Volume 26,  Issue 4,  Oct. 2001 Page(s):726 - 741 
# Items view:
#   Volume: 26  Issue: 4  Oct 2001 
#   Page(s): 726-741

_searchdataext = r"""
(
    \s* ^\d\d*\. \s* (?P<title>.*?)$
    \s* ^(?P<authors>.*?)$   
    \s* ^(?P<publication>.*?)$
    \s* ^(?P<pubdetails>.*?)$
    \s* (?:^(?P<pages>Page\(s\):\s*.*?)$)?
    \s* (?:^Digital[ ]Object[ ]Identifier\s*(?P<doi>.*?)$)?
    (?P<abstract>.*?)
    \s* ^Abstract .*? IEEE?[ ] (?P<entrytype>.*?)$
)
"""

# ^(?:Volume[ ](\d*))?,? \s* (?:Issue[ ](\d*))?,?(?:((?:\d\d-\d\d)?[ ].*?\d\d\d\d))?[ ]Page\(s\):(.*)$
# ^(?:Volume[ ](\d*))?,? \s* (?:Issue[ ](\d*))?,?(?:((?:\d\d-\d\d)?[ ].*?\d\d\d\d))?\s*Page\(s\):(.*)$
_searchdataext_re = re.compile(_searchdataext,
                            re.VERBOSE | re.DOTALL | re.MULTILINE)


# regexp for extracting publication details.
_pubdetailsext = r"""
(
    ^(?:Volume:?[ ](?P<volume>\d*))?,? \s* 
    (?:Issue:?[ ](?P<issue>\d*))?,? \s*
    (?:Part:?[ ](?P<part>\d*))?,? \s*
    (?:(?P<pubdate>(?:\d*-?\d*)?\s*.*?\d\d\d\d))?\s*
    Page\(s\):\s*(?P<pages>.*)$
)
"""

_pubdetailsext_re = re.compile(_pubdetailsext, re.VERBOSE | re.MULTILINE )


def extractIEEEXData(src):
    """Extract data from Search view and Abstract view
    
    Returns a list of dict suitable for use by IEEEXConverter
    """
    # process src so it's easier to extract information
    # remove whitespace and split into lines
    lines = [line.strip() for line in src.split('\n')]
    # we have to detect if it's a search or abstract view
    
    # look for a abstract view signature first
    i = 0
    is_abstractview = False
    # TODO: Find a more elegant solution
    for line in lines:
        if line.startswith('This paper appears in'): 
            is_abstractview = True
            break
        i += 1
    if is_abstractview:
        # make it possible to extract data even if the whole web page is
        # selected. This is done by removing superfluous text before the title    
        i -= 5
        if i < 0 : i = 0
        src = "\n".join(lines[i:])
        entry = extractAbstractViewData(src)
        return [entry]
    else:
        # is it a search view?
        # There is no need to remove superfluous text since each entry
        # in the search view starts with a number: 1. Title
        src = "\n".join(lines)
        entries = extractSearchViewData(src)
        return entries

def extractAbstractViewData(src):
    """Extract data from IEEEXplore Abstract(Plus) view
    
    Return data as a dictionary
    """
    # extract bibliographic data
    m = _dataext_re.match(src)
    if not m: 
        return None
    entry = m.groupdict()
    # extract index terms
    if src.find('Available to subscribers and IEEE members') < 0:
        m = _indextermsext_re.search(src)
        if m: 
            entry.update(m.groupdict())
            
    fields = _tagdata_re.findall(entry['datapart'])
    if not fields: return None
    for (tag, data) in fields:
        entry[tag.lower()] = data
    del entry['datapart']
    
    # have to fix some of the fields first
    entry['authors'] = [item.strip() for item in 
                        entry.get('authors','').split('  ')]
    # the volume field is combined with the issue field like this:
    # Volume: 13, Issue: 6
    vol = entry.get('volume')
    if vol:
        pieces = vol.split(',')
        if len(pieces) == 2:
            entry['volume'] = pieces[0].strip()
            pieces = pieces[1].split(':')
            if len(pieces) == 2:
                entry[pieces[0].lower().strip()] = pieces[1].strip() 
                
    # keyword extraction
    # check if keywords are available
    # TODO: Handle missing ci keywords!            
    entry['keywords'] = [item.strip() for item in 
                         entry.get('ci','').split('  ')]
    if entry['keywords']== '': 
        del entry['keywords']
    elif entry['keywords'][0] == 'Not Available':
        del entry['keywords']
    if entry.has_key('ci'): del entry['ci']
    return entry
    
def extractSearchViewData(src):
    """Extract data from IEEEXplore search view
    
    Return data as a list of dictionaries
    """
    # The strategy is to transform the data to a format that can be used
    # directly by IEEEXConverter.
    entrymap = {'JNL':'article','CNF':'inproceedings'}
    
    # extract bibliographic data
    matches = _searchdataext_re.finditer(src)
    if not matches: 
        return None
    entries = []
    for m in matches:
        entry = []
        entry = m.groupdict()
        # have to fix some of the fields first
        # split authors field
        entry['authors'] = [item.strip() for item in 
                        entry.get('authors','').split(';') if item]
        #del entry['abstract']
        renameKey(entry, 'publication','this paper appears in')
        renameKey(entry,'doi','digital object identifier')
        entry['entrytype'] = entrymap.get(entry['entrytype'],'misc')
        
        # There's a difference between the search view and the
        # "View Selected Items" view. In the latter, publication
        # details are split in two lines. It's therefore necessary to
        # join them again before the publication details are extracted. 
        if entry.get('pages',''):
            pubdetails = entry['pubdetails']
            entry['pubdetails'] = pubdetails+' '+entry['pages']
        # extract publication details
        pubm = _pubdetailsext_re.match(entry["pubdetails"])
        if pubm:
            entry.update(pubm.groupdict())
            renameKey(entry,'pages','on page(s)')
            renameKey(entry,'pubdate','publication date')
        else:
            #print entry
            # TODO: Should raise an error
            pass
        entries.append(entry)

    return entries

class IEEEXConverter(converterbase.ToBibTeXConverter):
    """Class for converting an IEEEXentry to BibTeX format
    """
    def __init__(self, entry, options = {}):
        # detect type
        if entry == None:
            print "Could not extract any data."
            self.entry_info = None
            return
        appears = entry.get('this paper appears in','').lower()
        if entry.has_key('entrytype'):
            entrytype = entry['entrytype']
        elif entry.has_key('location') or entry.has_key('meeting date'):
            entrytype = 'inproceedings'
        elif appears.find('conference') >= 0:
             #or appears.find('proceedings') >=0:
            entrytype = 'inproceedings'
        elif entry.get('issn',''):
            entrytype = 'article'
        else:
            entrytype = 'inproceedings'
        converterbase.ToBibTeXConverter.__init__(self, entry, 
                bibtexbib.entrymap.get(entrytype,bibtexbib.misc_entry),options)
        self.bibtexentry.type = entrytype
           
    def abstractHandler(self):
        abstract =  self.getField('abstract')
        if abstract:
            self.bibtexentry.append('abstract', abstract.strip())

    def addressHandler(self):
        address = self.getField('location')
        if address:
            self.bibtexentry.append('address',address)

    def authorHandler(self):
        author_fields = ['authors']
        authors, used_fields = getall(self.entry, author_fields)
        if authors:
            self.bibtexentry.append('author',authors)
            self.unused_fields -= used_fields

    def booktitleHandler(self):
        # The booktitle for a proceeding this is a tricky field
        bt_fields = ['this paper appears in']
        booktitle, used_fields = getfirst(self.entry,bt_fields)
        
        if booktitle:
            #pieces = booktitle.split(',')
            #if len(pieces) == 2:
            #    booktitle = "%s %s" % (pieces[1].strip(), pieces[0].strip())
            self.bibtexentry.append('booktitle',booktitle)
            self.unused_fields -= used_fields

    def doiHandler(self):
        # ieeex have changed their field name
        doi = self.getField('digital object identifier')
        if doi:
            self.bibtexentry.append('doi',doi)

    def ISSNHandler(self):
        issn = self.getField('issn')
        if issn:
            self.bibtexentry.append('ISSN',issn)

    def journalHandler(self):
        journal = self.getField('this paper appears in')
        if journal:
            # a typical journal field look like this: 
            #   Networking, IEEE/ACM Transactions on 
            # try to write it like:
            #   IEEE/ACM Transactions on Networking 
            pieces = journal.split(',')
            if len(pieces) == 2:
                journal = "%s %s" % (pieces[1].strip(), pieces[0].strip())
            self.bibtexentry.append('journal',journal)

    def keywordsHandler(self):
        # Elsevier operates with controlled keywords and user keywords. 
        # How should this be handled? Separate into two different fields?
        keywords, used_fields = getall(self.entry, ['keywords'])
        if keywords:
            self.bibtexentry.append('keywords',keywords)
            self.unused_fields -= used_fields
            
    def monthHandler(self):
        # Examples:
        #   11-14 Sept. 2000
        #   29 July-1 Aug. 2005
        #print self.entry
        pd = self.getField('publication date')
        #print pd
        if pd:
            month = None
            months = extStrs(pd)
            if len(months) == 1:
                # Convert to BibTeX month abbrev.
                if _monthmap.has_key(months[0]):
                    month = '@'+_monthmap[months[0]]
                    
                    if self.options.get('includedate',False):
                        dates = extPages(pd)
                        if dates:
                            month += ' # " %s--%s, "' % (dates[0],dates[1])
                        else:
                            days = extNums(pd)
                            if len(days) == 2:
                                month += ' # " %s, "' % (days[0])
            elif len(months) == 2:
                #print months
                month1 = _monthmap.get(months[0], months[0])
                month2 = _monthmap.get(months[1], months[1])
                days = extNums(pd)
                if len(days) > 2 and self.options.get('includedate',False):
                    startday = days[0]
                    endday = days[1]
                    month = '@'+ '%s # " %s--" # %s # " %s, "' % \
                                (month1, startday, month2, endday)
                else:
                    month = '@'+ month1 + ' # "/" # ' + month2
            if month:
                self.bibtexentry.append('month',month)
            else:
                #self.bibtexentry.append('month','unknown')
                pass

    def numberHandler(self):
        num = extNum(self.getField('issue'));
        if num: self.bibtexentry.append('number',num)

    def pagesHandler(self):
        pages = self.getField('on page(s)')
        pages = extPages(pages)
        if pages:
            pages = "%s--%s" % pages
            self.bibtexentry.append('pages', pages)

    def titleHandler(self):
        # get title
        title, used_fields = getfirst(self.entry, ['title'],'')
        if title:
            self.bibtexentry.append('title',title)
            self.unused_fields -= used_fields
    
    def volumeHandler(self):
        vol = extNum(self.getField('volume'));
        if vol: self.bibtexentry.append('volume',vol)
            
    def yearHandler(self):
        year = extYear(self.getField('publication date'))
        if year:
            self.bibtexentry.append('year',year)
    
def convertFromString(src, options = {}):
    """Extract data from src and return a list of BibTeX entries.
    
    Input:
        scr - Raw text usually pasted from the clipboard.
        options - A dict with various conversion options.
    Output:
        A list of BibTeXEntry classes.
    """
    ieeexentries = extractIEEEXData(src)
    if not ieeexentries: return None
    entries = []
    for entry in ieeexentries:
        t = IEEEXConverter(entry, options)
        t.convert()
        entries.append(t.bibtexentry)
    return entries

def convertFromDict(src, options = {}):
    t = IEEEXConverter(src, options)
    t.convert()
    return t.bibtexentry
    
    
if __name__ == '__main__':
    import testdata
    #src = "\n".join([line.strip() for line in testdata.ieee7.split('\n')])
    #m = _dataext_re.search(src)
    #d =  m.groupdict()
    options = {'ieeebst':True,'braceuppercase' : True, 'generatebibkey':True}
    d = extractIEEEXData(testdata.ieeesearch2)
    b = convertFromString(testdata.ieee4, options)
    #e = convertFromString(testdata.testdata5) 
    print b[0].toString()
    
    
    
